#!/bin/bash
execshell(){
#—————————————————传参变量—————————————————

service_key=$1           #进程关键字
service_name=$2          #模块名(最好带中文解释)
log_path=$3              #所有具体日志路径，可多个,请空格隔开
stop_command=$4          #停止命令
start_command=$5         #启动命令
log_dir=$6               #关键日志路径
log_keyswd=$7            #日志进程启动关键字:sessionCreate
status_command=$8        #状态查询命令（如有）


#固定参数
upuser=user
threaddump_command="/usr/local/jdk1.8.0_45/bin/jstack"
heapdump_command="/usr/local/jdk1.8.0_45/bin/jmap"
node_ip=$(/sbin/ifconfig | grep '10.252' | awk -F:   '{print $2}' | awk '{print $1}')         #节点ipnode_ip
save_path="/tmp/collect_msg"
msg_save_path="${save_path}/${node_ip}_${service_name%(*}_$(date "+%Y%m%d_%H%M%S")"    #信息存放目录
running_log_path="/tmp/"   #运行日志目录
mkdir -p ${msg_save_path}  > /dev/null 2>&1
mkdir -p ${running_log_path}   > /dev/null 2>&1
########################信息收集########################

#———————————————————————————————dump收集函数———————————————————————————————
	collect_dump()
	{
	        pid=$(ps -ef |  grep  java  |grep   "$service_key\s"  2> /dev/null | grep -v "grep" | wc -l)
	        if [ $pid -eq 1 ];then
	                echo -e "==========开始dump信息收集:$(date)=========="
	                echo   $(date)   >>    ${msg_save_path}/threaddump.txt
	                threaddump_old_hash=$(md5sum    ${msg_save_path}/threaddump.txt | awk '{print $1}')
	                sudo -u  $upuser   $threaddump_command  -F  $(ps -ef | grep "$service_key\s" | grep -v "grep" | awk '{print $2}' ) >>    ${msg_save_path}/threaddump.txt  2>&1
	                threaddump_new_hash=$(md5sum    ${msg_save_path}/threaddump.txt | awk '{print $1}')
	                echo "——————————————————————————————————————————————————————————————"         >>      ${msg_save_path}/threaddump.txt
	                if [ $threaddump_old_hash != $threaddump_new_hash ];then
	                        echo -e "$node_ip节点${service_name}_threaddump信息收集成功"
	                        dumpa=1
	                else
	                        echo -e "${node_ip}节点${service_name}_threaddump信息收集失败"
	                        dumpa=0
	                fi

	                echo   $(date)   >>    ${msg_save_path}/heapdump.txt
	                heapdump_old_hash=$(md5sum  ${msg_save_path}/heapdump.txt  | awk '{print $1}' )
	                sudo -u  $upuser  $heapdump_command  -heap  $(ps -ef | grep "$service_key\s"| grep -v "grep" | awk '{print $2}' )  >>    ${msg_save_path}/heapdump.txt    2>&1
	                heapdump_new_hash=$(md5sum  ${msg_save_path}/heapdump.txt  | awk '{print $1}' )
	                echo "——————————————————————————————————————————————————————————————"         >>    ${msg_save_path}/heapdump.txt
	                if [ $heapdump_old_hash != $heapdump_new_hash ];then
	                        echo -e "$node_ip节点${service_name}_heapdump信息收集成功"
	                        dumpb=1
	                else
	                        echo -e "${node_ip}节点${service_name}_heapdump信息收集失败"
	                        dumpb=0
	                fi
	        fi

	}
#———————————————————————————————主机信息收集函数———————————————————————————————
	collect_hostmsg()
	{
	        echo -e "==========开始主机信息收集:$(date)=========="

	        # cpu
	        echo   $(date)   >>  ${msg_save_path}/cpu.txt
	        cpu_old_hash=$(md5sum    ${msg_save_path}/cpu.txt  | awk '{print $1}'  )
	        uptime           >>  ${msg_save_path}/cpu.txt       2>&1
	        vmstat 1 5       >>  ${msg_save_path}/cpu.txt       2>&1
	        top -bn 3 | grep 'top -' -A 10 | tail -n 11 >>  ${msg_save_path}/cpu.txt       2>&1
	        cpu_new_hash=$(md5sum    ${msg_save_path}/cpu.txt  | awk '{print $1}'  )
	        echo "——————————————————————————————————————————————————————————————"         >>  ${msg_save_path}/cpu.txt
	        [ $cpu_old_hash != $cpu_new_hash ] && echo -e "$node_ip节点cpu信息收集成功" ||  echo -e "${node_ip}节点cpu信息收集失败"

	        # 内存
	        echo   $(date)   >>  ${msg_save_path}/memory.txt
	        memory_old_hash=$(md5sum    ${msg_save_path}/memory.txt | awk '{print $1}' )
	        free   -m        >>  ${msg_save_path}/memory.txt      2>&1
	        memory_new_hash=$(md5sum    ${msg_save_path}/memory.txt | awk '{print $1}' )
	        echo "——————————————————————————————————————————————————————————————"         >>  ${msg_save_path}/memory.txt
	        [ $memory_old_hash != $memory_new_hash ] && echo -e "$node_ip节点内存信息收集成功" ||  echo -e "${node_ip}节点内存信息收集失败"

	        # 磁盘
	        echo   $(date)   >>  ${msg_save_path}/disk.txt
	        disk_old_hash=$(md5sum    ${msg_save_path}/disk.txt  | awk '{print $1}' )
	        df -hP           >>  ${msg_save_path}/disk.txt        2>&1
	        df -iP           >>  ${msg_save_path}/disk.txt        2>&1
	        disk_new_hash=$(md5sum    ${msg_save_path}/disk.txt  | awk '{print $1}' )
	        echo "——————————————————————————————————————————————————————————————"         >>  ${msg_save_path}/disk.txt
	        [ $disk_old_hash != $disk_new_hash ] && echo -e "$node_ip节点磁盘信息收集成功" ||  echo -e "${node_ip}节点磁盘信息收集失败"

	        # 网络
	        echo   $(date)   >>  ${msg_save_path}/net.txt
	        net_old_hash=$(md5sum    ${msg_save_path}/net.txt    | awk '{print $1}')
	        netstat -anultp  >>  ${msg_save_path}/net.txt         2>&1
	        net_new_hash=$(md5sum    ${msg_save_path}/net.txt    | awk '{print $1}' )
	        echo "——————————————————————————————————————————————————————————————"         >>  ${msg_save_path}/net.txt
	        [ $net_old_hash != $net_new_hash ] && echo -e "$node_ip节点网络信息收集成功" ||  echo -e "${node_ip}节点网络信息收集失败"

	        #其他
	        #echo   $(date)   >>  ${msg_save_path}/other.txt
	}
#———————————————————————————————日志收集函数———————————————————————————————
	collect_log()
	{
	                if [[  $log_path !=  "" ]];then
	                echo -e "==========开始日志信息收集:$(date)=========="
	                for  i  in   $log_path
	                        do
	                        log=$(echo $i | awk -F/ '{print $NF}')
	                        sudo -u $upuser  tail  -10000   $i  >  ${msg_save_path}/${log}_$(date "+%Y-%m-%d_%H:%M").txt
	                        echo -e "${node_ip}节点${service_name}_${log}日志信息收集成功"
	                        done
	            else
	                echo -e "日志信息收集失败"
	            fi
	}
#———————————————————————————————运行日志函数———————————————————————————————
    running_log()
    {
            if [[ $dumpa == 1 ]] &&  [[ $dumpb == 1 ]] ||  [[ $pid == 0 ]];then
            		sh /home/itm6/send.sh   "一键故障信息收集/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/成功"  2   >  /dev/null  2>&1
            else
            		sh /home/itm6/send.sh   "一键故障信息收集/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/失败"  2   >  /dev/null  2>&1                        
            fi
    }
#———————————————————————————————信息集中函数———————————————————————————————
	send_msg()
	{
	        sh /home/itm6/send.sh   ${msg_save_path}  1 >  /dev/null  2>&1
	}

#———————————————————————————————清理函数———————————————————————————————
	clear_msg()
	{
	        file_del_list=$(find  ${save_path}   -type f  -mtime  +30)
	        for i in  ${file_del_list}
	                do
	                        echo  "清理过期信息$i"
	                        rm -rf  $i
	                done
	}
########################服务恢复########################

#———————————————————————————————停止服务函数———————————————————————————————
    step_stop()
    {
    		#ntime=$(date)
    		#threadexist=$(ps -ef|grep -v grep |grep $service_key |wc -l)
    		#检测进程是否存在，不存在则直接跳到启动服务。存在则输出进程存在，检查是否需要重启，并退出脚本。
			#if [[ ${threadexist} -ne 0 ]];then
        	#	echo "检测到进程已存在！无需重启！"
        	#	sh /home/itm6/send.sh   "故障自愈_服务恢复/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/进程存在不执行重启"  2   >  /dev/null  2>&1
        	#	#exit
			#else
			#	echo "检测到服务已不存在！开始重启！"
			#fi
            echo -e "==========开始停止服务:$ntime=========="
            sudo -u $upuser  $stop_command    > /dev/null  2>&1   # 停服务
            #sudo -u $upuser  kill -9  $(ps -ef | grep $service_key | grep -v "grep" | awk '{print $2}' )   > /dev/null  2>&1      # 停服务
            sleep 10 
            nowstate=$( [ $(sudo -u $upuser ps -ef | grep "$service_key\s" | grep -v  "grep"|wc -l )  -eq  0  ] && echo not running || echo running )            # 服务状态
            #nowstate=$(sudo -u $upuser $status_command)            # 服务状态，需要传查状态命令
            echo "-$node_ip-$service_name执行停止"
            echo "-$node_ip-$service_name停止后状态为：$nowstate"
            

            threadexist=$(ps -ef|grep -v grep |grep "$service_key\s"|wc -l)      # 判断服务是否停止
            if [   $threadexist -gt 0  ];
            then
                 echo "there previous thread is $(ps -ef|grep -v grep |grep $service_key|awk '{print $2}')"
                 sudo -u $upuser kill -9 $(ps -ef|grep -v grep |grep "$service_key\s"|awk '{print $2}')
                 echo "再次$service_name停止成功"
            fi
            before_stop=$(sudo -u $upuser wc -l $log_dir | awk '{print $1}')    #  停止后日志行数 需要传参数
    }
#———————————————————————————————启动服务函数———————————————————————————————
    step_restart()
    {
            ntime=$(date)
            echo -e "\n=========开始启动服务：$ntime========="
            sudo -u $upuser $start_command
            nowstate2=$( [ $(sudo -u $upuser ps -ef | grep "$service_key\s" | grep -v  "grep"|wc -l )  -eq  0  ] && echo not running || echo running )            # 服务状态
            #nowstate2=$(sudo -u $upuser $status_command)                     # 服务状态，需要传查状态命令

            threadexist=$(ps -ef|grep -v grep |grep "$service_key\s"|wc -l)
            if [  $threadexist -eq 0  ];then
                 sudo -u $upuser $start_command
                 threadexist=$(ps -ef|grep -v grep |grep "$service_key\s"|wc -l)
                 if [  $threadexist -eq 0  ];then
                    echo "$service_name启动失败,请核查！"
                            echo "3" > /tmp/breakpoint.txt
                            sh /home/itm6/send.sh   "故障自愈_服务恢复/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/失败"  2   >  /dev/null  2>&1
                            exit                                        # 启动失败直接停止脚本，产生断点索引
                 else
                    echo "$service_name进程已二次启动"
                 fi
            fi

            ###########

            check=0                                           # 循环
            countime=0                                        # 时间长度
            while [ $check -eq 0 ]
            do

	            sleep 20
	            countime=$(($countime+20))
	            after_start=$(sudo -u $upuser wc -l $log_dir | awk '{print $1}')     #  启动后日志行数 需要传参数

	            if [ $countime -gt 360 ];then
	                    echo '启动失败，超时>6MIN，请核查日志。'
	                    echo "3" > /tmp/breakpoint.txt
	                    sh /home/itm6/send.sh   "故障自愈_服务恢复/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/失败"  2   >  /dev/null  2>&1
	                    log=$(echo $log_dir | awk -F/ '{print $NF}')
				        sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir >> ${msg_save_path}/${log}_start_$(date "+%Y-%m-%d_%H:%M").txt   # 另存启动日志
					    echo -e "${node_ip}节点${service_name}_${log}日志启动信息收集成功"
					    send_msg             # 发送收集信息到 
	                    exit                                              # 启动失败直接停止脚本，产生断点索引
	            else
	                    if [ $before_stop -eq $after_start ];then
	                        echo "已耗时$countime"',日志未刷新，请核查！'
	                        echo "3" > /tmp/breakpoint.txt
	                        sh /home/itm6/send.sh   "故障自愈_服务恢复/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/失败"  2   >  /dev/null  2>&1
	                        exit                                          # 启动失败直接停止脚本，产生断点索引
	                    fi
	                    echo "启动中，已耗时${countime}s"
	            fi
	            ################
	            # sleep 20;check=1                                              # 日志关键字无法生效使用
	            check=$(sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir | grep "$log_keyswd" | wc -l)  # 日志关键字无法生效使用注销使用上面
            done

            #sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir | grep "$log_keyswd"

            #check_error=$(sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir | grep -i 'error' | wc -l) # 报错行数  # 可注销

            #sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir | grep "error"  | wc -l  # 输出错误信息
            #保留启动日志
            log=$(echo $log_dir | awk -F/ '{print $NF}')
            sudo -u $upuser sed -n ''$(($before_stop+1))','$after_start'p' $log_dir >> ${msg_save_path}/${log}_start_$(date "+%Y-%m-%d_%H:%M").txt   # 另存启动日志
	        echo -e "${node_ip}节点${service_name}_${log}日志启动信息收集成功"
	        send_msg             # 发送收集信息到
	        
	        
	        
            
            #############

            ntime=$(date)
            echo -e "\n==========检测启动完毕：$ntime=========="
            echo "-$node_ip-$service_name已重启，当前状态为$nowstate2"
            sh /home/itm6/send.sh   "故障自愈_服务恢复/$(date '+%Y-%m-%d %H:%M:%S')/${service_name}/${node_ip}/成功"  2   >  /dev/null  2>&1
            sleep 1

            ntime=$(date)
            echo -e "\n==========启动日志验证：$ntime=========="
            if [[ $check_error -ne 0 ]];then
                    echo "启动日志存在${check_error}行error日志，请核查！位于日志$(($before_stop+1)),${after_start}行"
            else
                    echo "启动日志无error级别日志"
            fi
	}

#—————————————————调用函数—————————————————
#collect_dump         # 调用收集dump函数
#collect_hostmsg      # 调用收集主机函数
#collect_log          # 调用收集log函数
#running_log     # 运行日志函数
#clear_msg            # 调用清理函数
#send_msg             # 发送收集信息到集中化主机  
#echo "==========信息收集结束:$(date)=========="
#echo "详细信息已存放在${node_ip}主机${msg_save_path}，并发送至"'10.252.168.198'"主机/tmp/collect_all/${msg_save_path}统一管理"
step_stop
step_restart        
}

#测试用例  /usr/local/epay/zf-ch-03-test
#execshell   "zf-ch-03-test"  "zf-ch-03-test_对端测试"  "/usr/local/epay/zf-ch-03-test/logs/service.log"       "/usr/local/epay/zf-ch-03-test/stop.sh"   "/usr/local/epay/zf-ch-03-test/startup.sh"  "/usr/local/epay/zf-ch-03-test/logs/service.log"   "sessionCreate"